Importing the data from drive¶
In [1]:
from google.colab import drive
drive.mount('/content/drive')
In [2]:
cd drive/MyDrive/Deep\ Learning\ Project
In [36]:
import pandas as pd
import numpy as np
import cv2
import matplotlib.pyplot as plt
import tensorflow as tf
import keras
from keras import Model
from tensorflow.keras.applications.resnet50 import ResNet50
import cv2
from keras.preprocessing import image
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions
import matplotlib.pyplot as plt
from keras.layers import GlobalMaxPooling2D
import warnings
import pickle
warnings.filterwarnings('ignore')
from keras.applications.vgg16 import VGG16
from keras.applications.inception_v3 import InceptionV3
from keras.models import Model
from keras.layers import Dense
from keras.layers import Flatten
from tensorflow.keras.applications import MobileNet
from tensorflow.keras.applications import DenseNet121
import matplotlib.pyplot as plt
from sklearn.feature_selection import SelectKBest
from sklearn import preprocessing
import random
import pickle
In [4]:
with open('images.npy', 'rb') as f:
images = np.load(f)
In [7]:
img_info_df = pd.read_csv("train_data.csv", index_col=0).reset_index()
In [35]:
img_info_df
Out[35]:
Category Wise Data¶
In [9]:
category_sum = img_info_df.groupby(['Image Category'], sort=True).size().reset_index(name='Count')
print("Total image categories:", len(category_sum))
category_sum.sort_values(by = 'Count', ascending=False)
Out[9]:
Creating a balanced train-test split¶
In [10]:
def trainTestSplit(category_sum):
categories = category_sum["Image Category"].tolist()
trainData = pd.DataFrame()
testData = pd.DataFrame()
for i in range(len(category_sum)):
count = category_sum.iloc[i]["Count"]
category = category_sum.iloc[i]["Image Category"]
data = img_info_df.loc[img_info_df["Image Category"]==category]
trainCount = int(0.8*count)
trainData = trainData.append(data.iloc[0:trainCount-1])
testData = testData.append(data.iloc[trainCount:])
return trainData, testData
In [11]:
trainData, testData = trainTestSplit(category_sum)
trainData = trainData.reset_index()
testData = testData.reset_index()
In [12]:
print(trainData.iloc[10000]['Image Paths'])
plt.imshow(images[trainData.iloc[10000]['index']])
Out[12]:
Deep Learning based existing models for feature extraction¶
In [13]:
def inceptionModel(height, width):
model = InceptionV3(weights='imagenet', include_top=False, input_shape = (height, width, 3))
model.trainable = False
output = GlobalMaxPooling2D()(model.outputs)
model = Model(inputs=model.inputs, outputs=output)
#model.summary()
return model
def mobileNet(height, width):
model = MobileNet(weights='imagenet', include_top=False, input_shape = (height, width, 3))
model.trainable = False
output = GlobalMaxPooling2D()(model.outputs)
model = Model(inputs=model.inputs, outputs=output)
model.summary()
return model
def resNet(height, width):
model = ResNet50(weights='imagenet', include_top=False, input_shape = (height, width, 3))
model.trainable = False
output = GlobalMaxPooling2D()(model.outputs)
model = Model(inputs=model.inputs, outputs=output)
model.summary()
return model
def vgg(height, width):
model = VGG16(weights='imagenet', include_top=False, input_shape = (height, width, 3))
model.trainable = False
output = GlobalMaxPooling2D()(model.outputs)
model = Model(inputs=model.inputs, outputs=output)
model.summary()
return model
def denseNet(height, width):
model = DenseNet121(weights='imagenet', include_top=False, input_shape = (height, width, 3))
model.trainable = False
output = GlobalMaxPooling2D()(model.outputs)
model = Model(inputs=model.inputs, outputs=output)
model.summary()
return model
def getFeatureVector(model, image):
featureVector = model.predict(image)
featureVector = featureVector.flatten()
return featureVector
Inception Model with Feature Extraction and Nearest Neighbors¶
In [14]:
print(trainData.iloc[10000]['Image Paths'])
# plt.imshow(images[trainData.iloc[10000]['index']])
image = images[trainData.iloc[10000]['index']]
plt.imshow(image)
model = inceptionModel(image.shape[0], image.shape[1])
image = np.expand_dims(image, axis = 0)
print(image.shape)
featureVector = getFeatureVector(model, image)
featureVector.shape
Out[14]:
In [16]:
def generateFeatures(model, data):
labels = []
features = []
ids = []
for index, row in data.iterrows():
ids.append(index)
image = images[data.iloc[index]['index']]
image = np.expand_dims(image, axis = 0)
label = row['Image Category']
featureVector = getFeatureVector(model, image)
features.append(featureVector)
labels.append(label)
return np.array(features), labels, ids
In [17]:
trainFeatures, trainLabels, trainIds = generateFeatures(model, trainData)
testFeatures, testLabels, testIds = generateFeatures(model, testData)
Nearest neighbours without added features¶
In [18]:
from sklearn.neighbors import NearestNeighbors
model_knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=5, n_jobs=-1)
model_knn.fit(trainFeatures)
test_data_neighbors = model_knn.kneighbors(testFeatures)
5 Recommended Images for each 5 Random images for Inception¶
In [19]:
test_result_indices = test_data_neighbors[1]
imgIds = [9,4,100,101,150]
for i in imgIds:
img_idx = testData.iloc[i]['index']
print("Original Image", i)
img_text = testData.iloc[i]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[img_idx])
plt.show()
for idx in test_result_indices[i]:
train_idx = trainData.iloc[idx]['index']
img_text = trainData.iloc[idx]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[train_idx])
plt.show()
VGG Model with Feature Extraction and Nearest Neighbors¶
In [42]:
print(trainData.iloc[10001]['Image Paths'])
# plt.imshow(images[trainData.iloc[10000]['index']])
image = images[trainData.iloc[10001]['index']]
plt.imshow(image)
model2 = vgg(image.shape[0], image.shape[1])
image = np.expand_dims(image, axis = 0)
print(image.shape)
featureVector = getFeatureVector(model2, image)
featureVector.shape
Out[42]:
In [43]:
def generateFeatures(model, data):
labels = []
features = []
ids = []
for index, row in data.iterrows():
ids.append(index)
image = images[data.iloc[index]['index']]
image = np.expand_dims(image, axis = 0)
label = row['Image Category']
featureVector = getFeatureVector(model, image)
features.append(featureVector)
labels.append(label)
return np.array(features), labels, ids
In [44]:
trainFeatures, trainLabels, trainIds = generateFeatures(model2, trainData)
testFeatures, testLabels, testIds = generateFeatures(model2, testData)
Nearest neighbours without added features¶
In [45]:
from sklearn.neighbors import NearestNeighbors
model_knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=5, n_jobs=-1)
model_knn.fit(trainFeatures)
test_data_neighbors = model_knn.kneighbors(testFeatures)
5 Recommended Images for each 5 Random images for VGG¶
In [46]:
test_result_indices = test_data_neighbors[1]
imgIds = [9,4,100,101,150]
for i in imgIds:
img_idx = testData.iloc[i]['index']
print("Original Image", i)
img_text = testData.iloc[i]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[img_idx])
plt.show()
for idx in test_result_indices[i]:
train_idx = trainData.iloc[idx]['index']
img_text = trainData.iloc[idx]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[train_idx])
plt.show()
In [47]:
saveInPickle([trainFeatures, trainLabels , trainIds], "vgg_train")
saveInPickle([testFeatures, testLabels, testIds], "vgg_test")
Resnet Model with Feature Extraction and Nearest Neighbors¶
In [28]:
print(trainData.iloc[10002]['Image Paths'])
# plt.imshow(images[trainData.iloc[10000]['index']])
image = images[trainData.iloc[10002]['index']]
plt.imshow(image)
model3 = resNet(image.shape[0], image.shape[1])
image = np.expand_dims(image, axis = 0)
print(image.shape)
featureVector = getFeatureVector(model3, image)
featureVector.shape
Out[28]:
In [29]:
def generateFeatures(model, data):
labels = []
features = []
ids = []
for index, row in data.iterrows():
ids.append(index)
image = images[data.iloc[index]['index']]
image = np.expand_dims(image, axis = 0)
label = row['Image Category']
featureVector = getFeatureVector(model, image)
features.append(featureVector)
labels.append(label)
return np.array(features), labels, ids
In [30]:
trainFeatures, trainLabels, trainIds = generateFeatures(model3, trainData)
testFeatures, testLabels, testIds = generateFeatures(model3, testData)
Nearest neighbours without added features¶
In [31]:
from sklearn.neighbors import NearestNeighbors
model_knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=5, n_jobs=-1)
model_knn.fit(trainFeatures)
test_data_neighbors = model_knn.kneighbors(testFeatures)
5 Recommended Images for each 5 Random images for Resnet¶
In [32]:
test_result_indices = test_data_neighbors[1]
imgIds = [9,4,100,101,150]
for i in imgIds:
img_idx = testData.iloc[i]['index']
print("Original Image", i)
img_text = testData.iloc[i]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[img_idx])
plt.show()
for idx in test_result_indices[i]:
train_idx = trainData.iloc[idx]['index']
img_text = trainData.iloc[idx]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[train_idx])
plt.show()
In [39]:
def loadFromPickle(pickleFile):
file = open(pickleFile,'rb')
pickleData = pickle.load(file)
file.close()
return pickleData
def saveInPickle(data, pickleFile):
file = open(pickleFile,"wb")
pickle.dump(data,file)
file.close()
In [40]:
saveInPickle([trainFeatures, trainLabels , trainIds], "resnet_train")
saveInPickle([testFeatures, testLabels, testIds], "resnet_test")
Resnet Model and VGG Model Ensemble as Feature Extraction and Nearest Neighbors¶
In [49]:
def generateEnsembleFeatures(Trainfeatures1, TestFeatures1, Trainfeatures2, TestFeatures2, weight1, weight2):
len1 = Trainfeatures1.shape[1]
len2 = Trainfeatures2.shape[1]
print(len1, len2)
classEncoder = preprocessing.LabelEncoder()
classEncoder.fit(trainLabels)
labels = classEncoder.transform(trainLabels)
if(len1 > len2):
xNew = SelectKBest(k=len2).fit(Trainfeatures1, labels)
Trainfeatures1 = xNew.transform(Trainfeatures1)
TestFeatures1 = xNew.transform(TestFeatures1)
else:
xNew = SelectKBest(k=len1).fit(Trainfeatures2, labels)
Trainfeatures2 = xNew.transform(Trainfeatures2)
TestFeatures2 = xNew.transform(TestFeatures2)
combinedTrainFeatures = Trainfeatures1*weight1 + Trainfeatures2*weight2
combinedTestFeatures = TestFeatures1*weight1 + TestFeatures2*weight2
return combinedTrainFeatures, combinedTestFeatures
In [50]:
def loadTrainFeatures(modelName):
trainSet = loadFromPickle(modelName+ "_train")
return trainSet[0], trainSet[1], trainSet[2]
def loadTestFeatures(modelName):
testSet = loadFromPickle(modelName+ "_test")
return testSet[0], testSet[1], testSet[2]
Setting the Ensemble model with weights¶
In [52]:
trainFeatures1, trainLabels , trainIds = loadTrainFeatures("resnet")
testFeatures1, testLabels , testIds = loadTestFeatures("resnet")
trainFeatures2, trainLabels , trainIds = loadTrainFeatures("vgg")
testFeatures2, testLabels , testIds = loadTestFeatures("vgg")
print(testFeatures1.shape, testFeatures2.shape)
trainFeatures, testFeatures = generateEnsembleFeatures(trainFeatures1, testFeatures1, trainFeatures2, testFeatures2,0.4,0.6)
Nearest neighbours without added features¶
In [53]:
from sklearn.neighbors import NearestNeighbors
model_knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=5, n_jobs=-1)
model_knn.fit(trainFeatures)
test_data_neighbors = model_knn.kneighbors(testFeatures)
5 Recommended Images for each 5 Random images for Ensemble Model¶
In [54]:
test_result_indices = test_data_neighbors[1]
imgIds = [9,4,100,101,150]
for i in imgIds:
img_idx = testData.iloc[i]['index']
print("Original Image", i)
img_text = testData.iloc[i]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[img_idx])
plt.show()
for idx in test_result_indices[i]:
train_idx = trainData.iloc[idx]['index']
img_text = trainData.iloc[idx]["Image Paths"].split("/")[1]
print(img_text)
plt.imshow(images[train_idx])
plt.show()
